package com.wifiin.springboot.webmagic.magic;

import com.wifiin.log.LoggerFactory;
import com.wifiin.springboot.webmagic.conversion.CreateTxtFile;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class SougouDictPageProcessor implements PageProcessor {
    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    public LoggerFactory.Logger logger = LoggerFactory.getLogger(SougouDictPageProcessor.class);
    private CreateTxtFile createTxtFile = new CreateTxtFile();

    @Override
    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    public void process(Page page) {
        String downUrlString = page.getHtml().css("div.dict_dl_btn").links().toString();// 下载路径链接
        if (downUrlString != null) {
            createTxtFile.readScelFile(downUrlString);
        }
        //从页面发现后续的url地址来抓取
        page.addTargetRequests(
                page.getHtml().links().regex("(https://pinyin.sogou\\.com/dict/[\\w\\-]+/[\\w\\-]+.*)").all());
    }

    @Override
    public Site getSite() {
        return site;
    }
}
